{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "db68f9ed-b3d6-4722-92da-d23681ceaab2",
   "metadata": {},
   "source": [
    "# Exercise 1\n",
    "\n",
    "## Reinforcement Learning\n",
    "\n",
    "\n",
    "---\n",
    "\n",
    "## Overview\n",
    "\n",
    "Welcome to this Excercise. We are now going to use our new skills to build our first Deep Learning Reinforcement Learning Model. \n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a73c7728",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: gymnasium in /opt/venv/lib/python3.10/site-packages (1.2.1)\n",
      "Requirement already satisfied: farama-notifications>=0.0.1 in /opt/venv/lib/python3.10/site-packages (from gymnasium) (0.0.4)\n",
      "Requirement already satisfied: cloudpickle>=1.2.0 in /opt/venv/lib/python3.10/site-packages (from gymnasium) (3.1.1)\n",
      "Requirement already satisfied: numpy>=1.21.0 in /opt/venv/lib/python3.10/site-packages (from gymnasium) (2.1.1)\n",
      "Requirement already satisfied: typing-extensions>=4.3.0 in /opt/venv/lib/python3.10/site-packages (from gymnasium) (4.12.2)\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m25.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install gymnasium"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "03286f4e-89e8-4615-9ac2-52a0ded7874e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "\n",
    "# Manually set the path relative to the py file's location that you want to import\n",
    "func_lib_path = os.path.abspath(os.path.join(os.getcwd(), '../../'))# Add the path to sys.path\n",
    "sys.path.append(func_lib_path)\n",
    "\n",
    "# Now you can import func_lib\n",
    "import func_lib\n",
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import gymnasium as gym\n",
    "from collections import defaultdict\n",
    "from gymnasium import spaces\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c99728be-57d6-4a52-80e9-ee83ce2e5726",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the custom gym environment for trading returns\n",
    "class ReturnEnv(gym.Env):\n",
    "    def __init__(self, df):\n",
    "        super(ReturnEnv, self).__init__()\n",
    "        \n",
    "        # Define the action and observation spaces\n",
    "        # - Action space is discrete with two possible actions: 0 or 1.\n",
    "        # - Observation space is a continuous box with shape (2,) to hold the return values.\n",
    "        self.action_space = spaces.Discrete(2)  # 0 or 1\n",
    "        self.observation_space = spaces.Box(low=-np.inf, high=np.inf, shape=(2,), dtype=np.float32)\n",
    "        \n",
    "        # Initialize the dataframe and step counter\n",
    "        # - Store the dataframe containing the trading data.\n",
    "        # - Initialize the current step to 0.\n",
    "        self.df = df\n",
    "        self.current_step = 0\n",
    "\n",
    "    def reset(self):\n",
    "        # Reset the environment to the initial state\n",
    "        # - Set the current step back to 0.\n",
    "        # - Return the initial state based on the first row of the dataframe.\n",
    "        self.current_step = 0\n",
    "        state = self.df.iloc[self.current_step][['1_d_returns']].values\n",
    "        return state\n",
    "        \n",
    "    def step(self, action):\n",
    "        # Retrieve the target return for the current step from the DataFrame.\n",
    "        target = self.df.iloc[self.current_step]['Target_Returns']\n",
    "        \n",
    "        # Calculate the reward based on the action taken.\n",
    "        # Reward is 1 if the action matches the target return, otherwise -1.\n",
    "        reward = 1 if action == target else -1\n",
    "        \n",
    "        # Move to the next step in the DataFrame.\n",
    "        self.current_step += 1\n",
    "        \n",
    "        # Check if the episode is done.\n",
    "        # An episode is considered done when all steps in the DataFrame have been processed.\n",
    "        done = self.current_step >= len(self.df)\n",
    "        \n",
    "        # Determine the next state to return.\n",
    "        # If the episode is not done, retrieve the '1_d_returns' value for the next step.\n",
    "        if not done:\n",
    "            next_state = self.df.iloc[self.current_step][['1_d_returns']].values\n",
    "        else:\n",
    "            # If the episode is done, return an array of zeros.\n",
    "            next_state = np.zeros(2)\n",
    "        \n",
    "        # Return the next state, the computed reward, the done flag, and an empty dictionary for additional information.\n",
    "        return next_state, reward, done, {}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "dab15e24-d1c7-4434-b261-a2d3333a2eac",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Q-learning Agent\n",
    "class QLearningAgent:\n",
    "    def __init__(self, action_space, state_space, alpha=0.1, gamma=0.99, epsilon=0.1):\n",
    "        # Initialize the Q-learning agent with parameters\n",
    "        # - Set the number of possible actions based on action_space.\n",
    "        # - Set the number of possible states based on state_space.\n",
    "        # - Initialize the learning rate (alpha) to 0.1 or another value as needed.\n",
    "        # - Initialize the discount factor (gamma) to 0.99 or another value as needed.\n",
    "        # - Initialize the exploration rate (epsilon) to 0.1 or another value as needed.\n",
    "        # - Create the Q-table as a defaultdict with zero-initialized arrays for each state-action pair.\n",
    "        self.action_space = action_space\n",
    "        self.state_space = state_space\n",
    "        self.alpha = alpha\n",
    "        self.gamma = gamma\n",
    "        self.epsilon = epsilon\n",
    "        self.q_table = defaultdict(lambda: np.zeros(action_space.n))\n",
    "    \n",
    "    def choose_action(self, state):\n",
    "        # Choose an action based on the exploration-exploitation trade-off\n",
    "        # - Generate a random number between 0 and 1.\n",
    "        # - If the random number is less than epsilon, choose a random action (exploration).\n",
    "        # - Otherwise, choose the action with the highest Q-value for the current state (exploitation).\n",
    "        if random.uniform(0, 1) < self.epsilon:\n",
    "            return self.action_space.sample()  # Explore by choosing a random action\n",
    "        else:\n",
    "            return np.argmax(self.q_table[str(state)])  # Exploit by choosing the best action based on Q-values\n",
    "    \n",
    "    def update(self, state, action, reward, next_state):\n",
    "        # Update the Q-values based on the agent's experience\n",
    "        # - Find the best action for the next state by checking Q-values.\n",
    "        # - Calculate the target Q-value (td_target) using the reward and the maximum Q-value for the next state.\n",
    "        # - Compute the temporal difference (TD) error as the difference between the target Q-value and the current Q-value.\n",
    "        # - Update the Q-value for the current state-action pair using the TD error and learning rate (alpha).\n",
    "        best_next_action = np.argmax(self.q_table[str(next_state)])  # Find the best action for the next state\n",
    "        td_target = reward + self.gamma * self.q_table[str(next_state)][best_next_action]  # Compute target Q-value\n",
    "        td_error = td_target - self.q_table[str(state)][action]  # Calculate TD error\n",
    "        self.q_table[str(state)][action] += self.alpha * td_error  # Update the Q-value for the state-action pair\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de1df900-6cc5-4fdb-a480-7d22fe0f0581",
   "metadata": {},
   "outputs": [],
   "source": [
    "def main():\n",
    "    # Example data\n",
    "    # Create a DataFrame 'df' with columns:\n",
    "    # - 'Target_Returns': Binary target indicating whether the return is positive (1) or not (0).\n",
    "    # - '1_d_returns': Daily return values.\n",
    "    df = pd.DataFrame({\n",
    "        'Target_Returns': [1, 0, 1, 1, 0, 1, 0, 1, 1, 0],\n",
    "        '1_d_returns': [0.062030, -0.038076, 0.050, 0.030, -0.020, 0.062030, -0.038076, 0.050, 0.030, -0.020]\n",
    "    })\n",
    "\n",
    "    # Create environment and agent\n",
    "    # Initialize the custom gym environment 'ReturnEnv' with the DataFrame 'df'.\n",
    "    env = ReturnEnv(df)\n",
    "    \n",
    "    # Initialize the Q-learning agent 'QLearningAgent' with:\n",
    "    # - The action space from the environment.\n",
    "    # - The observation space from the environment.\n",
    "    agent = QLearningAgent(env.action_space, env.observation_space)\n",
    "\n",
    "    # Training loop\n",
    "    # Set the number of episodes for training.\n",
    "    n_episodes = 1000\n",
    "    \n",
    "    # Loop through each episode.\n",
    "    for episode in range(n_episodes):\n",
    "        # Reset the environment to start a new episode.\n",
    "        state = env.reset()\n",
    "        done = False\n",
    "        \n",
    "        # Run the episode until it's done.\n",
    "        while not done:\n",
    "            # Choose an action based on the current state using the agent.\n",
    "            action = agent.choose_action(state)\n",
    "            \n",
    "            # Take the chosen action and get the next state, reward, and done flag.\n",
    "            next_state, reward, done, _ = env.step(action)\n",
    "            \n",
    "            # Update the agent with the state, action, reward, and next state.\n",
    "            agent.update(state, action, reward, next_state)\n",
    "            \n",
    "            # Move to the next state.\n",
    "            state = next_state\n",
    "\n",
    "    # Indicate that training is finished.\n",
    "    print(\"Training finished.\")\n",
    "\n",
    "    # Evaluation\n",
    "    # Reset the environment to start the evaluation.\n",
    "    state = env.reset()\n",
    "    done = False\n",
    "    step = 1\n",
    "\n",
    "    # Evaluate the agent's performance.\n",
    "    while not done:\n",
    "        # Choose an action based on the current state using the agent.\n",
    "        action = agent.choose_action(state)\n",
    "        \n",
    "        # Take the chosen action and get the next state, reward, and done flag.\n",
    "        next_state, reward, done, _ = env.step(action)\n",
    "        \n",
    "        # Print detailed information for each step, including:\n",
    "        # - Action taken.\n",
    "        # - Target return for the current step.\n",
    "        # - Reward received.\n",
    "        # - Step number.\n",
    "        # - Current state.\n",
    "        # - Next state.\n",
    "        print(f\"Action: {action}, Target Return: {env.df['Target_Returns'].values[step-1]}, Reward: {reward}, Step: {step}, State: {state}, Next State: {next_state}\")\n",
    "        \n",
    "        # Move to the next state.\n",
    "        state = next_state\n",
    "        step += 1\n",
    "\n",
    "# Run the main function if this script is executed directly.\n",
    "if __name__ == \"__main__\":\n",
    "    main()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
